<?php
/**
 * STEP 1
 * 更新：2017-10-17 /36521.html
 */

// LC_ALL=en_US.UTF-8 /usr/bin/casperjs capture.js '{$url}'
include_once('../lib/config.php');
include_once('../lib/Db.class.php');
include_once('../lib/Model.class.php');
include_once('../lib/function.php');

//列出已下载文件数量
/*
$downpath = './download/同步盘/qisuu_download/武侠仙侠/';
$dirs = scandir($downpath);
foreach($dirs as $dir) {
    if($dir == '.' || $dir == '..' || $dir == '.DS_Store' || !is_dir("{$downpath}{$dir}")) continue;

    $files = scandir("{$downpath}{$dir}");

    echo '[quote]' . $dir . "[/quote]\n\n";
    echo(implode(', ', $files));

    echo "\n\n";
}

exit;*/

$menu_urls = [
    '最近更新' => 'https://www.qisuu.com/s/new/index_1.html',/*
    '玄幻奇幻' => 'https://www.qisuu.com/soft/sort01/index.html',
    '武侠仙侠' => 'https://www.qisuu.com/soft/sort02/index.html',
    '女频言情' => 'https://www.qisuu.com/soft/sort03/index.html',
    '现代都市' => 'https://www.qisuu.com/soft/sort04/index.html',
    '历史军事' => 'https://www.qisuu.com/soft/sort05/index.html',
    '游戏竞技' => 'https://www.qisuu.com/soft/sort06/index.html',
    '科幻灵异' => 'https://www.qisuu.com/soft/sort07/index.html',
    '美文同人' => 'https://www.qisuu.com/soft/sort08/index.html',
    '剧本教程' => 'https://www.qisuu.com/soft/sort09/index.html',
    '名著杂志' => 'https://www.qisuu.com/soft/sort010/index.html',*/
];

clearstatcache();
$page = 1;
$overduetime = 3600; //一个小时内不重复抓取

//读取所有列表页
foreach ($menu_urls as $key => $_url) {
    while (true) {
        if($page >= 5) {
            break;
        }

        if($page > 1) {
            $url = str_replace('_1.html', "_{$page}.html", $_url);
        } else {
            $url = $_url;
        }

        $filename  = QISUU_NEW . 'menulist/' . urlencode($url);
        $filemtime = file_exists($filename) ? intval(filemtime($filename)) : 0;
        echo2($filename . ' mtime ' .date('Y-m-d H:i:s', $filemtime), 'file.mtime');
        if(file_exists($filename) && filesize($filename)>5000 && (!stristr($url, '/new/index') || ($filemtime+$overduetime>time()))) {
            echo2("$filename file_exists", 'file.exists');
            $page++;
            continue;
        }

        //--proxy=127.0.0.1:1080 --proxy-type=socks5
        $shell = "LC_ALL=en_US.UTF-8 /usr/bin/casperjs qisuu.js '{$url}' '".QISUU_NEW."menulist/'";
        echo2($shell, 'shell.command');
        $result = shell_exec($shell);
        echo2($result, 'shell.result');

        if(strstr($result, 'STATUS_ERROR') || $page >= 5) {
            $page = 1;
            break;
        }
        elseif (stristr($url, '/new/index') || 1) {
            $page++;
        }
    }
}

$redis = new Redis();  
$ret = $redis->connect("localhost","6379");  //php客户端设置的ip及端口
if(!$ret) die('redis connction faild');
$redis->auth(REDIS_PASS);
$redis->select(10);

$lsize = $redis->lSize('QISUU_INFO_LINK');
echo2($lsize . ' in queue', 'QISUU.PHP');

if($lsize > 10000) {
    exit;
}

//读取所有详情地址压入redis
$old = 0;
$new = 0;
$files = scandir(QISUU_NEW . 'menulist/');
foreach ($files as $file) {
    if(!stristr($file, '.html')) {
        continue;
    }

    $contents = file_get_contents(QISUU_NEW . 'menulist/'.$file);
    $contents = cut('<div class="listBox">', '</ul>', $contents);

    //echo $contents; break;
    preg_match_all('/<a href="(.+?)"><img/i', $contents, $matches);
    //print_r($matches[1]); break;
    
    foreach ($matches[1] as $item) {
        if(!stristr($item, 'https://')) {
            $item = "https://www.qisuu.com" . $item;
        }

        //36207之前不需要
        $temp = explode('/', $item);
        $temp = end($temp);
        $temp = str_ireplace('.html', '', $temp);
        $temp = str_ireplace('shtml', '', $temp);
        $temp = (int) $temp;
        if($temp <= 36600) {
            echo2("Skipped id:{$temp}, item:{$item}", 'QISUU.PHP');
            continue;
        }

        $filename  = QISUU_NEW . 'htmls/' . urlencode($item);
        if(file_exists($filename)) {
            $old++;
            echo2("$old $filename file_exists");
            continue;
        }

        $redis->lPush('QISUU_INFO_LINK', $item);

        $new++;
        echo2($new . ' ' . $item . ' push', 'QISUU.PHP');
    }
}
$redis->lPush('stats_qisuu', "[".date('Y-m-d H:i:s')."], old:$old, new:$new");
echo2("old:$old, new:$new", 'QISUU.PHP');

sleep(10);

echo2('sleep exit', 'QISUU.PHP');

